{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 深度学习第二周作业\n",
    "目标：mnist数据集准确率打到99.5%\n",
    "\n",
    "以下代码基于TensorFlow 1.12版本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入工具包\n",
    "\n",
    "import tensorflow.contrib.slim as slim\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-256f13207c74>:4: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data/mnist/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data/mnist/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting data/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/mnist/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# 读取数据\n",
    "\n",
    "data_dir = 'data/mnist/'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义参数\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 784], name='x')  # 图片占位符\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])   # 真值占位符\n",
    "\n",
    "# 格式化图片数据\n",
    "with tf.name_scope('reshape'):\n",
    "    x_image = tf.reshape(x, shape=[-1, 28, 28, 1], name='x_image')\n",
    "\n",
    "learning_rate = tf.constant(0.9, name='lr')   # 学习率\n",
    "is_training = tf.placeholder(tf.bool, name='mode')  # batch normalization 是否训练模式\n",
    "decay = tf.constant(0.9, name='decay')  # batch normalization 衰减\n",
    "\n",
    "bn_params = dict(is_training=is_training, decay=decay) # batch normalization 参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 建立模型\n",
    "\n",
    "'''\n",
    "使用arg_scope统一管理参数\n",
    "卷积层默认padding为SAME，默认stride为[1, 1]\n",
    "池化层默认padding为VALID， 默认stride为[2, 2]\n",
    "'''\n",
    "with slim.arg_scope([slim.conv2d, slim.fully_connected],   # 参数作用于卷积层和全连接层\n",
    "                    normalizer_fn=slim.batch_norm,         # batch normalization\n",
    "                    normalizer_params=bn_params,           # 设置batch normalization参数\n",
    "                    weights_initializer=slim.xavier_initializer()  # 权重初始化方法使用xavier效果比较好\n",
    "                    ):\n",
    "    # 第一层卷积使用128个卷积核输出数据shape[-1, 28, 28, 128]\n",
    "    net = slim.conv2d(x_image, 128, [5, 5], scope='conv1')  \n",
    "    # 第一层池化降低数据高和宽 输出数据shape[-1, 14, 14, 128]\n",
    "    net = slim.max_pool2d(net, [2, 2], scope='pool1')\n",
    "    # 第二层卷积使用256个卷积核输出数据shape[-1, 28, 28, 256]\n",
    "    net = slim.conv2d(net, 256, [5, 5], scope='conv2')\n",
    "    # 第二层池化降低数据高和宽 输出数据shape[-1, 7, 7, 256]\n",
    "    net = slim.max_pool2d(net, [2, 2], scope='pool2')\n",
    "    # 降低数据维度到二维\n",
    "    net = slim.flatten(net, scope='flatten')\n",
    "    # 全连接层采用100个神经元\n",
    "    net = slim.fully_connected(net, 100, scope='fc1')\n",
    "    # dropout层，默认keep_prob为0.5，随机使一半神经元死亡，防止过拟合。\n",
    "    net = slim.dropout(net, is_training=is_training, scope=\"dropout\")\n",
    "    # 全连接层采用10个神经元，这里不用batch normalization，因为已经是最终层。\n",
    "    net = slim.fully_connected(net, 10, activation_fn=None, normalizer_fn=None, scope='fc2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-8d2c37e5136a>:4: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 定义损失函数 采用交叉熵损失\n",
    "# 因为我们已经使用了batch normalization和dropout来防止过拟合，所以这里不再使用l2损失。\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(logits=net, labels=y_)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义训练步骤\n",
    "update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "with tf.control_dependencies(update_ops):   # 这里使得train_step依赖于update_ops使得batch normalization能正常工作。\n",
    "    # train_step = tf.train.AdamOptimizer(0.01).minimize(cross_entropy)\n",
    "    # 这里还是采用梯度下降的方式训练，因为AdamOptimizer效果并不稳定。\n",
    "    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义正确率函数\n",
    "correct_prediction = tf.equal(tf.argmax(y_, 1), tf.argmax(net, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化变量\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess = tf.Session()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:  1 lr:  0.9 accuracy:  0.9828 cross entropy:  0.050458986\n",
      "epoch:  2 lr:  0.9 accuracy:  0.9791 cross entropy:  0.07075897\n",
      "epoch:  3 lr:  0.089999996 accuracy:  0.9917 cross entropy:  0.02380039\n",
      "epoch:  4 lr:  0.089999996 accuracy:  0.9927 cross entropy:  0.023443708\n",
      "epoch:  5 lr:  0.089999996 accuracy:  0.9927 cross entropy:  0.02149044\n",
      "epoch:  6 lr:  0.089999996 accuracy:  0.9932 cross entropy:  0.02086541\n",
      "epoch:  7 lr:  0.089999996 accuracy:  0.9916 cross entropy:  0.02521532\n",
      "epoch:  8 lr:  0.009 accuracy:  0.9931 cross entropy:  0.019081626\n",
      "epoch:  9 lr:  0.009 accuracy:  0.9935 cross entropy:  0.022359382\n",
      "epoch:  10 lr:  0.0009 accuracy:  0.9935 cross entropy:  0.019940583\n",
      "epoch:  11 lr:  0.0009 accuracy:  0.9935 cross entropy:  0.020503208\n",
      "epoch:  12 lr:  9e-05 accuracy:  0.993 cross entropy:  0.022531375\n",
      "epoch:  13 lr:  9e-06 accuracy:  0.9935 cross entropy:  0.019346982\n",
      "epoch:  14 lr:  9e-06 accuracy:  0.9939 cross entropy:  0.020386433\n",
      "epoch:  15 lr:  9.0000003e-07 accuracy:  0.9946 cross entropy:  0.017007891\n",
      "epoch:  16 lr:  9.0000003e-07 accuracy:  0.9938 cross entropy:  0.020568253\n",
      "epoch:  17 lr:  9.000001e-08 accuracy:  0.993 cross entropy:  0.021337852\n",
      "epoch:  18 lr:  9.000001e-09 accuracy:  0.9944 cross entropy:  0.020490354\n",
      "epoch:  19 lr:  9.000001e-09 accuracy:  0.9944 cross entropy:  0.020638535\n",
      "epoch:  20 lr:  9.000001e-10 accuracy:  0.9935 cross entropy:  0.02153726\n",
      "epoch:  21 lr:  9.000001e-11 accuracy:  0.9941 cross entropy:  0.022044718\n",
      "epoch:  22 lr:  9.000001e-12 accuracy:  0.9943 cross entropy:  0.019895606\n",
      "epoch:  23 lr:  9.000001e-12 accuracy:  0.9945 cross entropy:  0.020877369\n",
      "epoch:  24 lr:  9.000001e-13 accuracy:  0.9941 cross entropy:  0.020315014\n",
      "epoch:  25 lr:  9.000001e-13 accuracy:  0.9945 cross entropy:  0.020012032\n",
      "epoch:  26 lr:  9.000001e-13 accuracy:  0.995 cross entropy:  0.018834533\n",
      "epoch:  27 lr:  9.000001e-13 accuracy:  0.9948 cross entropy:  0.019988734\n",
      "epoch:  28 lr:  9.0000015e-14 accuracy:  0.9942 cross entropy:  0.018032808\n",
      "epoch:  29 lr:  9.0000015e-14 accuracy:  0.9948 cross entropy:  0.018459374\n",
      "epoch:  30 lr:  9.000002e-15 accuracy:  0.9945 cross entropy:  0.019870328\n",
      "epoch:  31 lr:  9.000002e-16 accuracy:  0.9941 cross entropy:  0.0224175\n",
      "epoch:  32 lr:  9.0000023e-17 accuracy:  0.9946 cross entropy:  0.020893894\n",
      "epoch:  33 lr:  9.0000023e-17 accuracy:  0.9949 cross entropy:  0.021295872\n",
      "epoch:  34 lr:  9.000002e-18 accuracy:  0.9949 cross entropy:  0.019769099\n",
      "epoch:  35 lr:  9.000002e-18 accuracy:  0.9943 cross entropy:  0.023212604\n",
      "epoch:  36 lr:  9.000002e-19 accuracy:  0.9948 cross entropy:  0.02120783\n",
      "epoch:  37 lr:  9.000002e-19 accuracy:  0.9951 cross entropy:  0.020383915\n",
      "epoch:  38 lr:  9.000002e-19 accuracy:  0.9946 cross entropy:  0.022495542\n",
      "epoch:  39 lr:  9.000002e-20 accuracy:  0.9952 cross entropy:  0.019778604\n",
      "epoch:  40 lr:  9.000002e-20 accuracy:  0.9943 cross entropy:  0.021322383\n",
      "epoch:  41 lr:  9.000002e-21 accuracy:  0.9948 cross entropy:  0.020977143\n",
      "epoch:  42 lr:  9.000002e-21 accuracy:  0.9936 cross entropy:  0.022441583\n",
      "epoch:  43 lr:  9.000003e-22 accuracy:  0.9944 cross entropy:  0.025625482\n",
      "epoch:  44 lr:  9.000003e-23 accuracy:  0.994 cross entropy:  0.024699776\n",
      "epoch:  45 lr:  9.000003e-23 accuracy:  0.9938 cross entropy:  0.026149442\n",
      "epoch:  46 lr:  9.000003e-24 accuracy:  0.9939 cross entropy:  0.024142291\n",
      "epoch:  47 lr:  9.000003e-24 accuracy:  0.9942 cross entropy:  0.023726696\n",
      "epoch:  48 lr:  9.000003e-24 accuracy:  0.995 cross entropy:  0.024087958\n",
      "epoch:  49 lr:  9.000003e-25 accuracy:  0.9945 cross entropy:  0.024338702\n",
      "epoch:  50 lr:  9.0000025e-26 accuracy:  0.9939 cross entropy:  0.025391554\n",
      "epoch:  51 lr:  9.000002e-27 accuracy:  0.9939 cross entropy:  0.022302337\n",
      "epoch:  52 lr:  9.000002e-27 accuracy:  0.9939 cross entropy:  0.025300639\n",
      "epoch:  53 lr:  9.000003e-28 accuracy:  0.9938 cross entropy:  0.026704673\n",
      "epoch:  54 lr:  9.0000024e-29 accuracy:  0.9939 cross entropy:  0.02407194\n",
      "epoch:  55 lr:  9.0000024e-29 accuracy:  0.9948 cross entropy:  0.021662109\n",
      "epoch:  56 lr:  9.0000024e-29 accuracy:  0.9945 cross entropy:  0.022283688\n",
      "epoch:  57 lr:  9.000002e-30 accuracy:  0.9947 cross entropy:  0.02209925\n",
      "epoch:  58 lr:  9.000002e-30 accuracy:  0.9952 cross entropy:  0.020971457\n",
      "epoch:  59 lr:  9.000002e-30 accuracy:  0.9957 cross entropy:  0.01933587\n",
      "epoch:  60 lr:  9.000002e-30 accuracy:  0.9956 cross entropy:  0.020118907\n",
      "epoch:  61 lr:  9.000002e-31 accuracy:  0.9945 cross entropy:  0.024631018\n",
      "epoch:  62 lr:  9.0000026e-32 accuracy:  0.995 cross entropy:  0.022220535\n",
      "epoch:  63 lr:  9.0000026e-32 accuracy:  0.9949 cross entropy:  0.022434432\n",
      "epoch:  64 lr:  9.000003e-33 accuracy:  0.9938 cross entropy:  0.025250696\n",
      "epoch:  65 lr:  9.000003e-34 accuracy:  0.9938 cross entropy:  0.024620317\n",
      "epoch:  66 lr:  9.000003e-34 accuracy:  0.9946 cross entropy:  0.023073563\n",
      "epoch:  67 lr:  9.000003e-34 accuracy:  0.9943 cross entropy:  0.023578193\n",
      "epoch:  68 lr:  9.0000026e-35 accuracy:  0.9945 cross entropy:  0.022782266\n",
      "epoch:  69 lr:  9.0000026e-35 accuracy:  0.9944 cross entropy:  0.02503209\n",
      "epoch:  70 lr:  9.000003e-36 accuracy:  0.9945 cross entropy:  0.023608338\n",
      "epoch:  71 lr:  9.000003e-36 accuracy:  0.9951 cross entropy:  0.021439286\n",
      "epoch:  72 lr:  9.000003e-36 accuracy:  0.9946 cross entropy:  0.022804834\n",
      "epoch:  73 lr:  9.000003e-37 accuracy:  0.9944 cross entropy:  0.020866247\n",
      "epoch:  74 lr:  9.000003e-37 accuracy:  0.9943 cross entropy:  0.023037907\n",
      "epoch:  75 lr:  9.0000036e-38 accuracy:  0.9949 cross entropy:  0.022393059\n",
      "epoch:  76 lr:  9.0000036e-38 accuracy:  0.9948 cross entropy:  0.022591088\n",
      "epoch:  77 lr:  0.0 accuracy:  0.9951 cross entropy:  0.022575846\n",
      "epoch:  78 lr:  0.0 accuracy:  0.9944 cross entropy:  0.023886466\n",
      "epoch:  79 lr:  0.0 accuracy:  0.9951 cross entropy:  0.021411505\n",
      "epoch:  80 lr:  0.0 accuracy:  0.9947 cross entropy:  0.024473462\n",
      "epoch:  81 lr:  0.0 accuracy:  0.995 cross entropy:  0.024508603\n",
      "epoch:  82 lr:  0.0 accuracy:  0.9947 cross entropy:  0.023338743\n",
      "epoch:  83 lr:  0.0 accuracy:  0.9946 cross entropy:  0.025878893\n",
      "epoch:  84 lr:  0.0 accuracy:  0.9944 cross entropy:  0.025352817\n",
      "epoch:  85 lr:  0.0 accuracy:  0.9944 cross entropy:  0.024773385\n",
      "epoch:  86 lr:  0.0 accuracy:  0.9939 cross entropy:  0.02581879\n",
      "epoch:  87 lr:  0.0 accuracy:  0.9947 cross entropy:  0.022898633\n",
      "epoch:  88 lr:  0.0 accuracy:  0.9947 cross entropy:  0.025983218\n",
      "epoch:  89 lr:  0.0 accuracy:  0.9951 cross entropy:  0.024276871\n",
      "epoch:  90 lr:  0.0 accuracy:  0.9944 cross entropy:  0.02634606\n",
      "epoch:  91 lr:  0.0 accuracy:  0.9949 cross entropy:  0.026217474\n",
      "epoch:  92 lr:  0.0 accuracy:  0.9944 cross entropy:  0.02524834\n",
      "epoch:  93 lr:  0.0 accuracy:  0.9947 cross entropy:  0.024784839\n",
      "epoch:  94 lr:  0.0 accuracy:  0.9949 cross entropy:  0.027707571\n",
      "epoch:  95 lr:  0.0 accuracy:  0.9947 cross entropy:  0.026873093\n",
      "epoch:  96 lr:  0.0 accuracy:  0.9945 cross entropy:  0.028697107\n",
      "epoch:  97 lr:  0.0 accuracy:  0.9939 cross entropy:  0.02727009\n",
      "epoch:  98 lr:  0.0 accuracy:  0.995 cross entropy:  0.027086947\n",
      "epoch:  99 lr:  0.0 accuracy:  0.9944 cross entropy:  0.026390543\n",
      "epoch:  100 lr:  0.0 accuracy:  0.995 cross entropy:  0.024643637\n"
     ]
    }
   ],
   "source": [
    "# 训练\n",
    "\n",
    "batch_size = 100\n",
    "n_batch = mnist.train.num_examples // batch_size\n",
    "\n",
    "loss = 0.0 # 记录损失\n",
    "for epoch in range(100):\n",
    "    for batch in range(n_batch):\n",
    "\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        sess.run(train_step,\n",
    "                 feed_dict={x: batch_xs,\n",
    "                            y_: batch_ys,\n",
    "                            is_training: True\n",
    "                            })\n",
    "    # 测试准确率\n",
    "\n",
    "    accuracy_value, \\\n",
    "    learning_rate_value, \\\n",
    "    cross_entropy_value = sess.run([accuracy,\n",
    "                                    learning_rate,\n",
    "                                    cross_entropy],\n",
    "                                   feed_dict={\n",
    "                                       x: mnist.test.images,\n",
    "                                       y_: mnist.test.labels,\n",
    "                                       is_training: False\n",
    "                                   })\n",
    "\n",
    "    print('epoch: ', epoch + 1,\n",
    "          'lr: ', learning_rate_value,\n",
    "          'accuracy: ', accuracy_value,\n",
    "          'cross entropy: ', cross_entropy_value\n",
    "          )\n",
    "\n",
    "    # 如果当前损失比上一轮损失大，说明学习率过大，减小学习率。\n",
    "    if loss != 0.0 and cross_entropy_value > loss:\n",
    "        learning_rate = learning_rate * 0.1\n",
    "\n",
    "    loss = cross_entropy_value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在第59个epoch之后，准确率达到最高值99.52% 之后一直在99.5%左右徘徊。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 结束语\n",
    "\n",
    "本次作业耗费了比较长的时间，查询了很多资料，对batch normalization和dropout有了进一步的理解，最终勉强使得准确率达到99.5%，然而本来打算用keras来写，结果得到的结果却不理想，估计是没有很好的掌握keras的使用方法。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
